import Runloop from '@runloop/api-client';
const client = new Runloop({
bearerToken: process.env['RUNLOOP_API_KEY'], // This is the default and can be omitted
});
const benchmarkJobView = await client.benchmarkJobs.retrieve('id');
console.log(benchmarkJobView.id);import os
from runloop_api_client import Runloop
client = Runloop(
bearer_token=os.environ.get("RUNLOOP_API_KEY"), # This is the default and can be omitted
)
benchmark_job_view = client.benchmark_jobs.retrieve(
"id",
)
print(benchmark_job_view.id)curl --request GET \
--url https://api.runloop.ai/v1/benchmark_jobs/{id} \
--header 'Authorization: Bearer <token>'<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.runloop.ai/v1/benchmark_jobs/{id}",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.runloop.ai/v1/benchmark_jobs/{id}"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.runloop.ai/v1/benchmark_jobs/{id}")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.runloop.ai/v1/benchmark_jobs/{id}")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"state": "initializing",
"create_time_ms": 123,
"job_source": {
"inline_yaml": "<string>",
"type": "harbor"
},
"job_spec": {
"scenario_ids": [
"<string>"
],
"agent_configs": [
{
"name": "<string>",
"type": "job_agent",
"agent_id": "<string>",
"model_name": "<string>",
"timeout_seconds": 123,
"kwargs": {},
"agent_environment": {
"environment_variables": {},
"secrets": {}
}
}
],
"orchestrator_config": {
"n_concurrent_trials": 123,
"n_attempts": 123,
"timeout_multiplier": 123,
"quiet": true
}
},
"failure_reason": "<string>",
"benchmark_outcomes": [
{
"benchmark_run_id": "<string>",
"agent_name": "<string>",
"n_completed": 123,
"n_failed": 123,
"n_timeout": 123,
"scenario_outcomes": [
{
"scenario_definition_id": "<string>",
"scenario_name": "<string>",
"state": "COMPLETED",
"scenario_run_id": "<string>",
"score": 123,
"duration_ms": 123,
"failure_reason": {
"exception_type": "<string>",
"exception_message": "<string>"
}
}
],
"model_name": "<string>",
"average_score": 123,
"duration_ms": 123
}
],
"in_progress_runs": [
{
"benchmark_run_id": "<string>",
"state": "running",
"start_time_ms": 123,
"agent_config": {
"type": "external_api",
"info": "<string>"
},
"duration_ms": 123
}
]
}[Beta] Get a previously created BenchmarkJob.
[Beta] Get a BenchmarkJob given ID.
import Runloop from '@runloop/api-client';
const client = new Runloop({
bearerToken: process.env['RUNLOOP_API_KEY'], // This is the default and can be omitted
});
const benchmarkJobView = await client.benchmarkJobs.retrieve('id');
console.log(benchmarkJobView.id);import os
from runloop_api_client import Runloop
client = Runloop(
bearer_token=os.environ.get("RUNLOOP_API_KEY"), # This is the default and can be omitted
)
benchmark_job_view = client.benchmark_jobs.retrieve(
"id",
)
print(benchmark_job_view.id)curl --request GET \
--url https://api.runloop.ai/v1/benchmark_jobs/{id} \
--header 'Authorization: Bearer <token>'<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.runloop.ai/v1/benchmark_jobs/{id}",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.runloop.ai/v1/benchmark_jobs/{id}"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.runloop.ai/v1/benchmark_jobs/{id}")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.runloop.ai/v1/benchmark_jobs/{id}")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"id": "<string>",
"name": "<string>",
"state": "initializing",
"create_time_ms": 123,
"job_source": {
"inline_yaml": "<string>",
"type": "harbor"
},
"job_spec": {
"scenario_ids": [
"<string>"
],
"agent_configs": [
{
"name": "<string>",
"type": "job_agent",
"agent_id": "<string>",
"model_name": "<string>",
"timeout_seconds": 123,
"kwargs": {},
"agent_environment": {
"environment_variables": {},
"secrets": {}
}
}
],
"orchestrator_config": {
"n_concurrent_trials": 123,
"n_attempts": 123,
"timeout_multiplier": 123,
"quiet": true
}
},
"failure_reason": "<string>",
"benchmark_outcomes": [
{
"benchmark_run_id": "<string>",
"agent_name": "<string>",
"n_completed": 123,
"n_failed": 123,
"n_timeout": 123,
"scenario_outcomes": [
{
"scenario_definition_id": "<string>",
"scenario_name": "<string>",
"state": "COMPLETED",
"scenario_run_id": "<string>",
"score": 123,
"duration_ms": 123,
"failure_reason": {
"exception_type": "<string>",
"exception_message": "<string>"
}
}
],
"model_name": "<string>",
"average_score": 123,
"duration_ms": 123
}
],
"in_progress_runs": [
{
"benchmark_run_id": "<string>",
"state": "running",
"start_time_ms": 123,
"agent_config": {
"type": "external_api",
"info": "<string>"
},
"duration_ms": 123
}
]
}Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Path Parameters
The BenchmarkJob ID.
Response
OK
A BenchmarkJobView represents a benchmark job that runs a set of scenarios entirely on runloop.
The ID of the BenchmarkJob.
The unique name of the BenchmarkJob.
The current state of the benchmark job.
initializing, queued, running, completed, failed, cancelled, timeout Timestamp when job was created (Unix milliseconds).
The source configuration that was used to create this job. Either Harbor YAML or benchmark definition reference.
- Option 1
- Option 2
- Option 3
Show child attributes
Show child attributes
The resolved job specification. Contains scenarios, agents, and orchestrator config.
Show child attributes
Show child attributes
Failure reason if job failed.
Detailed outcome data for each benchmark run created by this job. Includes per-agent results and scenario-level details.
Show child attributes
Show child attributes
Benchmark runs currently in progress for this job. Shows runs that have not yet completed.
Show child attributes
Show child attributes
Was this page helpful?
